Final Assignment¶
| Project | Analysis of Chicago Real Estate Market |
|---|---|
| Subject | Machine Learning |
| Assignment | Final Assignment |
| Team Name | Group 4 |
| Team Members | Mary Kate Krusenoski Yunhao Dou Phill Betts Zijun Wu |
Loading Libraries¶
In [1]:
import numpy as np
import pandas as pd
import pickle
import datetime as dt
import missingno as msno
import sklearn.model_selection
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler
import math
from sklearn.metrics import make_scorer
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
%matplotlib inline
from sklearn import preprocessing
from sklearn.preprocessing import OneHotEncoder
from sklearn.metrics import mean_squared_error,mean_absolute_error
from sklearn.linear_model import Ridge
# suppressing pesky warnings
import warnings
warnings.filterwarnings("ignore")
warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.simplefilter(action='ignore', category=Warning)
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from wordcloud import WordCloud
from autoimpute.imputations import SingleImputer, MultipleImputer, MiceImputer
import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt
import datetime
import folium
from folium.plugins import FastMarkerCluster
from folium.plugins import HeatMap
import json
Preprocessing¶
Loading Data¶
In [2]:
# Read the file containing the data from redfin
df = pd.read_excel ('Sold_05222022.xlsx')
#Chicago Crime 2021
dfCrime = pd.read_csv ('ChiCrime.csv')
Removing Extra Columns¶
In [3]:
df.head()
Out[3]:
In [4]:
# drop the columns that are not needed for building the model
df=df.drop(['Source.Name','SALE TYPE','STATE OR PROVINCE','STATUS',
'NEXT OPEN HOUSE START TIME',
'NEXT OPEN HOUSE END TIME',
'FAVORITE',
'INTERESTED',
'MLS#',
'STATUS',
'STATE OR PROVINCE',
'URL (SEE http://www.redfin.com/buy-a-home/comparative-market-analysis FOR INFO ON PRICING)',#has missing data
'SOURCE',
'$/SQUARE FEET'
], axis=1)
#keep address for identification later
# remove special character as well as spaces in between column names
# removing them helps later in processing
df.columns = df.columns.str.replace(' ', '')
In [5]:
#For testing imputation methods
df1 = df.copy()
In [6]:
df = df1.copy()
Combining Crimes Data¶
In [7]:
dfCrime.info()
In [8]:
#october 31 2020 - 2021
dfCrime.head()
Out[8]:
In [9]:
#Converting Community Area from numerical data to neighborhood to merge with Redfin LOCATION data
dfCrime["Community Area"] = dfCrime["Community Area"].astype(str)
Communitydict = {
'1.0':'Rogers Park',
'2.0':'West Ridge',
'3.0':'Uptown',
'4.0':'Lincoln Square',
'5.0':'North Center',
'6.0':'Lakeview',
'7.0':'Lincoln Park',
'8.0':'Near North Side',
'9.0':'Edison Park',
'10.0':'Norwood Park',
'11.0':'Jefferson Park',
'12.0':'Forest Glen',
'13.0':'North Park',
'14.0':'Albany Park',
'15.0':'Portage Park',
'16.0':'Irving Park',
'17.0':'Dunning',
'18.0':'Montclare',
'19.0':'Belmont Cragin',
'20.0':'Hermosa',
'21.0':'Avondale',
'22.0':'Logan Square',
'23.0':'Humboldt Park',
'24.0':'West Town',
'25.0':'Austin',
'26.0':'West Garfield Park',
'27.0':'East Garfield Park',
'28.0':'Near West Side',
'29.0':'North Lawndale',
'30.0':'South Lawndale',
'31.0':'Lower West Side',
'32.0':'Loop',
'33.0':'Near South Side',
'34.0':'Armour Square',
'35.0':'Douglas',
'36.0':'Oakland',
'37.0':'Fuller Park',
'38.0':'Grand Boulevard',
'39.0':'Kenwood',
'40.0':'Washington Park',
'41.0':'Hyde Park',
'42.0':'Woodlawn',
'43.0':'South Shore',
'44.0':'Chatham',
'45.0':'Avalon Park',
'46.0':'South Chicago',
'47.0':'Burnside',
'48.0':'Calumet Heights',
'49.0':'Roseland',
'50.0':'Pullman',
'51.0':'South Deering',
'52.0':'East Side',
'53.0':'West Pullman',
'54.0':'Riverdale',
'55.0':'Hegewisch',
'56.0':'Garfield Ridge',
'57.0':'Archer Heights',
'58.0':'Brighton Park',
'59.0':'McKinley Park',
'60.0':'Bridgeport',
'61.0':'New City',
'62.0':'West Elsdon',
'63.0':'Gage Park',
'64.0':'Clearing',
'65.0':'West Lawn',
'66.0':'Chicago Lawn',
'67.0':'West Englewood',
'68.0':'Englewood',
'69.0':'Greater Grand Crossing',
'70.0':'Ashburn',
'71.0':'Auburn Gresham',
'72.0':'Beverly',
'73.0':'Washington Heights',
'74.0':'Mount Greenwood',
'75.0':'Morgan Park',
'76.0':'O Hare',
'77.0':'Edgewater'
}
dfCrime["Community Area"] = dfCrime["Community Area"].map(Communitydict)
In [10]:
#Crime by type
dfCrime['Primary Type'].value_counts()
Out[10]:
In [11]:
#We want to aggregate total crimes for the year per Community Area
crimeCountLocation = pd.DataFrame(dfCrime['Community Area'].value_counts().reset_index().values, columns=['Community Area', "AggregateCrime"])
crimeCountindex = crimeCountLocation.sort_index(axis = 0, ascending=True)
crimeCountindex
Out[11]:
In [12]:
#Removing Locations with unmappable characters
df['LOCATION']=df['LOCATION'].str.replace('CHI - ','')
In [13]:
df = pd.merge(df, crimeCountindex, how='left', left_on='LOCATION', right_on='Community Area')
df.head()
Out[13]:
In [14]:
#Dropping merged location column
df=df.drop(['Community Area'],axis = 1)
Data Mapping¶
In [15]:
#Reviewing all property types. We will only include top 3 in our analysis
df['PROPERTYTYPE'].value_counts()
Out[15]:
In [16]:
# converting the sold date field to a proper date time format
df['SOLDDATE'] = pd.to_datetime(df['SOLDDATE'])
df['SOLDDATE']=pd.DatetimeIndex(df['SOLDDATE']).year
df.dropna(subset = ['SOLDDATE'], inplace=True)
df['SOLDDATE']=df['SOLDDATE'].astype(int)
In [17]:
df['CITY'] = df.CITY.str.upper()
In [18]:
#Removing Arlington Heights
df = df[df['CITY'] != 'ARLINGTON HEIGHTS']
In [19]:
df['CITY'].value_counts()
Out[19]:
In [20]:
#For listing on Cities
df1 = df.copy()
In [21]:
#Mapping data to numerical values
mapping_dict = {"PROPERTYTYPE":{
"Condo/Co-op": 1,
"Single Family Residential": 2,
"Townhouse": 3,
"Multi-Family (2-4 Unit)": 4,
"Multi-Family (5+ Unit)": 5,
"Other": 6,
"Vacant Land": 7,
"Parking":8
},
"CITY":{
'CHICAGO' : 1,
'OAK PARK' : 0,
'OAK LAWN' : 0,
'BERWYN' : 0,
'BURBANK' : 0,
'EVERGREEN PARK' : 0,
'ELMWOOD PARK' : 0,
'FOREST PARK' : 0,
'CICERO' : 0,
'RIVER FOREST' : 0,
'HOMETOWN' : 0,
'STICKNEY' : 0,
'LINCOLNWOOD' : 0,
'HARWOOD HEIGHTS' : 0,
'BRIDGEVIEW' : 0,
'SKOKIE' : 0,
'NORRIDGE' : 0,
'RIVERSIDE' : 0,
'NORTH RIVERSIDE' : 0,
'RIVER GROVE' : 0,
'BURNHAM' : 0,
'FOREST VIEW' : 0,
'RIVERDALE' : 0,
'CALUMET PARK' : 0,
'MELROSE PARK' : 0,
'NOTTINGHAM PARK' : 0,
'LAWNDALE' : 0,
'MAYWOOD' : 0,
'NILES' : 0},
"SOLDDATE":{
2016: 1,
2017: 2,
2018: 3,
2019: 4,
2020: 5,
2021: 6,
2022: 7
}
}
df = df.replace(mapping_dict)
#changing numerical datatypes
df['PROPERTYTYPE'] = df['PROPERTYTYPE'].astype('str')
df['CITY'] = df['CITY'].astype('str')
df['LOCATION'] = df['LOCATION'].astype('str')
df['SOLDDATE'] = df['SOLDDATE'].astype('int')
catmap = preprocessing.LabelEncoder()
df['PROPERTYTYPE'] = catmap.fit_transform(df['PROPERTYTYPE'].values)
df['LOCATION'] = catmap.fit_transform(df['LOCATION'].values)
df['CITY'] = catmap.fit_transform(df['CITY'].values)
df['ZIPORPOSTALCODE'] = catmap.fit_transform(df['ZIPORPOSTALCODE'].values)
df['YEARBUILT'] = catmap.fit_transform(df['YEARBUILT'].values)
In [22]:
df.info()
In [23]:
#Removing Multi-Family, Vacant lots, or Other property types
df = df[df['PROPERTYTYPE'] < 4]
In [24]:
df.shape
Out[24]:
In [25]:
df.head()
Out[25]:
Removing Duplicate rows¶
In [26]:
# Since the data collection was done by multiple team members,
# there is a chance there are duplicates
df.drop_duplicates(keep='first',inplace=True)
df1.drop_duplicates(keep='first',inplace=True)
Listings per City¶
In [27]:
fig = plt.figure()
fig, ax = plt.subplots()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(True)
ax.spines['bottom'].set_visible(False)
feq=df1['CITY'].value_counts().nlargest(5).sort_values(ascending=True)
feq.plot.barh(figsize=(10, 10), color="red", width=0.9)
plt.title("Top 5 Cities (Listings)", fontsize=20)
for i, v in enumerate(feq):
ax.text(v + 75, i - 0.1, str(v), color="black", fontweight='bold')
ax.set_xticks([])
plt.show()
#fig.savefig('./Photos/Top5.png', transparent=True)
Removing Null Values¶
In [28]:
df.shape
Out[28]:
In [29]:
#Drop Blank Latitudes that account for a large missing portion of the data, including price
df.dropna(subset = ['LATITUDE'], inplace=True)
In [30]:
total = df.isnull().sum().sort_values(ascending=False)
percent_1 = df.isnull().sum()/df.isnull().count()*100
percent_2 = (round(percent_1, 1)).sort_values(ascending=False)
missing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%'])
missing_data.head(33)
Out[30]:
In [31]:
df.groupby(['SOLDDATE']).mean()
Out[31]:
In [32]:
#median price over time
table = pd.pivot_table(df, values='PRICE',columns=['SOLDDATE'], aggfunc=np.median)
table
Out[32]:
Imputing Null Values¶
In [33]:
# correlation heat map setup for seaborn
def corr_chart(df_corr):
corr=df_corr.corr()
#screen top half to get a triangle
top = np.zeros_like(corr, dtype=np.bool)
top[np.triu_indices_from(top)] = True
fig=plt.figure()
fig, ax = plt.subplots(figsize=(12,12))
sns.heatmap(corr, mask=top, cmap='coolwarm',
center = 0, square=True,
linewidths=.5, cbar_kws={'shrink':.5},
annot = True, annot_kws={'size': 9}, fmt = '.3f')
plt.xticks(rotation=45) # rotate variable labels on columns (x axis)
plt.yticks(rotation=0) # use horizontal variable labels on rows (y axis)
plt.title('Correlation Heat Map')
plt.savefig('plot-corr-map.pdf',
bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b',
orientation='portrait', papertype=None, format=None,
transparent=True, pad_inches=0.25, frameon=None)
np.set_printoptions(precision=3)
In [34]:
# with correlation matrix/heat map
corr_chart(df_corr = df)
In [35]:
#Imputing SQ Feet based on KNN nearest neighbors
import sklearn.impute
from sklearn.impute import SimpleImputer, KNNImputer
knniputer = KNNImputer()
knniputer.fit(df[["SQUAREFEET"]])
df["SQUAREFEET"] = knniputer.transform(df[["SQUAREFEET"]])
df["SQUAREFEET"] = df["SQUAREFEET"].apply(lambda x: int(x))
df["SQUAREFEET"].unique()
Out[35]:
In [36]:
#impute HOA/ Month with zero.
#Assuming those without HOA's are likely houses without HOA's
df['HOA/MONTH'] = df['HOA/MONTH'].fillna(0)
In [37]:
#Dropping Days on Market and Lot Size since there is no correlation nor impact
df=df.drop(['DAYSONMARKET', 'LOTSIZE'],axis=1)
In [38]:
#impute aggregate crime with the median
df['AggregateCrime']=df['AggregateCrime'].fillna(df['AggregateCrime'].median())
In [39]:
total = df.isnull().sum().sort_values(ascending=False)
percent_1 = df.isnull().sum()/df.isnull().count()*100
percent_2 = (round(percent_1, 1)).sort_values(ascending=False)
missing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%'])
missing_data.head(33)
Out[39]:
In [40]:
df.dropna(inplace=True)
In [41]:
#Longitude has error '0' in data and two large homes with 10000 square feet
df = df[df['LONGITUDE'] != 0]
df = df[df['SQUAREFEET'] < 9999]
In [42]:
#post clean data this is the final size of our dataset
df.shape
Out[42]:
Quick Visualization¶
Quick visualization to see obvious gaps in data
In [43]:
m=folium.Map([41.85, -87.68],zoom_start=11)
HeatMap(df[['LATITUDE','LONGITUDE']].dropna(),radius=8,gradient={0.25:'blue',0.5:'purple',0.75:'orange',1.0:'red'}).add_to(m)
display(m)
In [44]:
# Removing zip codes related to Arlington heights
df=df[~df.ZIPORPOSTALCODE.isin([60004, 60005, 60006, 60008, 60056, 60095])]
In [45]:
df[df['CITY'] == 'Arlington Heights']['ZIPORPOSTALCODE'].value_counts()
Out[45]:
In [46]:
m=folium.Map([41.85, -87.68],zoom_start=11)
HeatMap(df[['LATITUDE','LONGITUDE']].dropna(),radius=8,gradient={0.25:'blue',0.5:'purple',0.75:'orange',1.0:'red'}).add_to(m)
display(m)
Other Visualization¶
In [47]:
lats2018 = df['LATITUDE'].tolist()
lons2018 = df['LONGITUDE'].tolist()
locations = list(zip(lats2018, lons2018))
map1 = folium.Map(location=[41.85, -87.68], zoom_start=11.5)
FastMarkerCluster(data=locations).add_to(map1)
map1
Out[47]:
Exploratory Data Analysis¶
Feature Analysis¶
In [48]:
df.info()
In [49]:
df.describe()
Out[49]:
In [50]:
df.plot(kind="scatter", x="SQUAREFEET", y="PRICE")
Out[50]:
In [51]:
df[df['SQUAREFEET']<600]['SQUAREFEET'].value_counts().sort_index()
#We want to remove houses listed with less than 176 as we can assume these are errors
df = df[df['SQUAREFEET']>=176]
In [52]:
plt.figure(figsize=(6,6))
sns.countplot(df['BEDS'])
plt.xlabel('Beds')
plt.ylabel('Count')
df.BEDS.value_counts().sort_index(ascending=True)
Out[52]:
In [53]:
plt.figure(figsize=(6,6))
sns.countplot(df['BATHS'])
plt.xlabel('Baths')
plt.ylabel('Count')
df.BATHS.value_counts().sort_index(ascending=True)
#These are likely skewed data
df = df[df['BATHS'] <14]
Price¶
In [54]:
filter_values = [ 50000, 500000,1000000,5000000 ]
fig = plt.figure()
fig, ax = plt.subplots()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(True)
ax.spines['bottom'].set_visible(False)
feq=df['PRICE'].value_counts(bins=filter_values).sort_values(ascending=True)
feq.plot.barh(figsize=(10,6), color="red", width=0.5)
plt.title("Price Range (in USD)", fontsize=20)
for i, v in enumerate(feq):
ax.text(v + 10, i - 0.1, str(v), color="black", fontweight='bold')
plt.tight_layout()
plt.show()
In [55]:
# This is common function to calcuate the Interquartile range
def outlier_lower_iqr(col_name, th1=0.25, th3=0.75):
quartile1 = df[col_name].quantile(th1)
quartile3 = df[col_name].quantile(th3)
iqr = quartile3 - quartile1
lower_limit = quartile1 - 1.5 * iqr
return lower_limit
def outlier_upper_iqr(col_name, th1=0.25, th3=0.75):
quartile1 = df[col_name].quantile(th1)
quartile3 = df[col_name].quantile(th3)
iqr = quartile3 - quartile1
upper_limit = quartile3 + 1.5 * iqr
return upper_limit
In [56]:
#Remove Price Outliers
num_list=[ "PRICE"]
for i in num_list:
df=df[df[i].between(outlier_lower_iqr(i), outlier_upper_iqr(i))]
In [57]:
fig = plt.figure()
ax = sns.distplot(df['PRICE'], color="red")
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['bottom'].set_visible(True)
ax.spines['left'].set_visible(True)
plt.title("Price", fontsize=20)
ax.set_xticks([])
ax.set_yticks([])
plt.show()
In [58]:
print("Skewness of PRICE: {}".format(df['PRICE'].skew()))
In [59]:
df1 =df.copy()
In [60]:
df1['PRICE'] = df1['PRICE'].apply(lambda x: math.log(x))
In [61]:
print("Skewness of PRICE: {}".format(df1['PRICE'].skew()))
#Data is more skewed when logged, will keep as is
Modeling¶
In [62]:
RANDOM_SEED = 50
In [63]:
# splitting data into test and train based on SOLDDATE(YEAR)
X = df.loc[:, ['SOLDDATE', 'PROPERTYTYPE', 'CITY', 'ZIPORPOSTALCODE', 'BEDS', 'BATHS', 'LOCATION', 'SQUAREFEET', 'YEARBUILT',
'HOA/MONTH', 'LATITUDE', 'LONGITUDE', 'AggregateCrime','PRICE']]
#Pre-covid prices should offset price increase in 2021/22 and show us pricing more in-line with
#what we would have expected before covid and inflation
train_data=X[X['SOLDDATE']<6]
test_data=X[X['SOLDDATE']>=6]
X_train = train_data.drop(['PRICE'], axis = 1)
X_test = test_data.drop(['PRICE'], axis = 1)
y_train = train_data["PRICE"]
y_test = test_data["PRICE"]
In [64]:
len(X_train), len(y_train), len(X_test), len(y_test), len(train_data)
Out[64]:
In [65]:
#Creating RMSE Scoring Function
def rmse(predict, actual):
predict = np.array(predict)
actual = np.array(actual)
distance = predict - actual
square_distance = distance ** 2
mean_square_distance = square_distance.mean()
score = np.sqrt(mean_square_distance)
return score
In [66]:
rmse_score = make_scorer(rmse, greater_is_better = False)
In [67]:
from sklearn.model_selection import cross_val_score
# function to get cross validation scores
def get_cv_scores(model):
scores = cross_val_score(model,
X_train,
y_train,
cv=5,
scoring='r2')
print('CV Mean: ', np.mean(scores))
print('STD: ', np.std(scores))
print('\n')
In [68]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
from sklearn import metrics
1. Linear Regression¶
In [69]:
from sklearn.linear_model import LinearRegression
# Train model
lin_reg = LinearRegression()
lin_reg.fit(X_train, y_train)
# get cross val scores
get_cv_scores(lin_reg)
Cross Validation score is low for linear regression try to improve using Ridge, Lasso, and Elastic Net.
In [70]:
prediction_linear = lin_reg.predict(X_test)
prediction_linear
Out[70]:
In [71]:
linear_mse = metrics.mean_squared_error(y_test, prediction_linear)
linear_mse
Out[71]:
In [72]:
linear_rmse = np.sqrt(linear_mse)
linear_rmse
Out[72]:
In [73]:
lin_reg.score(X_train, y_train)
Out[73]:
In [74]:
rmse(y_test, prediction_linear)
Out[74]:
2. Ridge Regression¶
In [75]:
from sklearn.linear_model import Ridge
# Train model with default alpha=1
ridge = Ridge(alpha=1).fit(X_train, y_train)
# get cross val scores
get_cv_scores(ridge)
CV mean still low
In [76]:
# find optimal alpha with grid search
alpha = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
param_grid = dict(alpha=alpha)
grid = GridSearchCV(estimator=ridge, param_grid=param_grid, scoring=rmse_score, verbose=1, n_jobs=-1)
grid_result = grid.fit(X_train, y_train)
print('Best Score: ', grid_result.best_score_)
print('Best Params: ', grid_result.best_params_)
even with best parameters RMSE score largely negative
In [77]:
ridge.score(X_train, y_train)
Out[77]:
In [78]:
y_pred_rd = ridge.predict(X_test)
rmse(y_test, y_pred_rd)
Out[78]:
3. Lasso Regression¶
In [79]:
from sklearn.linear_model import Lasso
# Train model with default alpha=1
lasso = Lasso(alpha=1).fit(X_train, y_train)
# get cross val scores
get_cv_scores(lasso)
CV mean similar to linear regression and ridge regression
In [80]:
# find optimal alpha with grid search
alpha = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
param_grid = dict(alpha=alpha)
grid = GridSearchCV(estimator=lasso, param_grid=param_grid, scoring=rmse_score, verbose=1, n_jobs=-1)
grid_result = grid.fit(X_train, y_train)
print('Best Score: ', grid_result.best_score_)
print('Best Params: ', grid_result.best_params_)
even with best parameters RMSE score largely negative
In [81]:
lasso.score(X_train, y_train)
Out[81]:
In [82]:
y_pred_ls = lasso.predict(X_test)
rmse(y_test, y_pred_ls)
Out[82]:
4. Elastic Net¶
In [83]:
from sklearn.linear_model import ElasticNet
# Train model with default alpha=1 and l1_ratio=0.5
elastic_net = ElasticNet(alpha=1, l1_ratio=0.5).fit(X_train, y_train)
# get cross val scores
get_cv_scores(elastic_net)
CV mean is even lower for elastic net than linear regression and ridge resgression
In [84]:
# find optimal alpha with grid search
alpha = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
l1_ratio = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
param_grid = dict(alpha=alpha, l1_ratio=l1_ratio)
grid = GridSearchCV(estimator=elastic_net, param_grid=param_grid, scoring=rmse_score, verbose=1, n_jobs=-1)
grid_result = grid.fit(X_train, y_train)
print('Best Score: ', grid_result.best_score_)
print('Best Params: ', grid_result.best_params_)
In [85]:
elastic_net.score(X_train, y_train)
Out[85]:
In [86]:
y_pred_en = elastic_net.predict(X_test)
rmse(y_test, y_pred_en)
Out[86]:
RMSE score is not optimal.
5. Decision Tree¶
In [87]:
criterion = ["mse", "mae"]
min_samples_split = [2, 5, 10, 20]
min_samples_leaf = [1, 20, 40, 100]
param_grid = {"criterion": criterion,
"min_samples_split": min_samples_split,
"min_samples_leaf": min_samples_leaf,
}
In [88]:
# Use GridSearchCV to search for best hyperparameters
# First create the base model to tune
decision_tree = DecisionTreeRegressor(random_state=RANDOM_SEED)
# Using 3 fold cross validation
grid_search = GridSearchCV(decision_tree, param_grid = param_grid, cv=3, scoring=rmse_score, n_jobs = -1)
grid_search.fit(X_train, y_train)
Out[88]:
In [89]:
final = grid_search.best_params_
print(final)
In [90]:
dt = DecisionTreeRegressor(criterion= 'mse', min_samples_leaf= 40, min_samples_split= 2,random_state=RANDOM_SEED)
In [91]:
dt.fit(X_train,y_train)
Out[91]:
In [92]:
y_pred_dt = dt.predict(X_test)
In [93]:
print('RMSE Score:',rmse(y_test, y_pred_dt))
In [94]:
dt.score(X_train, y_train)
Out[94]:
In [95]:
get_cv_scores(dt)
In [96]:
features = X_train.columns
importances = dt.feature_importances_
indices = np.argsort(importances)
fig = plt.figure(figsize=(25,25))
fig, ax = plt.subplots()
#plt.figure( figsize=(20,20) )
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(True)
ax.spines['bottom'].set_visible(False)
plt.barh(range(len(indices)), importances[indices], align='center', color="red")
plt.yticks(range(len(indices)), [features[i] for i in indices]);
plt.xlabel('Relative Importance')
plt.tight_layout()
plt.show()
6. Random Forest¶
In [97]:
# splitting data into test and train based on SOLDDATE(YEAR)
X = df.loc[:, ['SOLDDATE', 'PROPERTYTYPE', 'CITY', 'ZIPORPOSTALCODE', 'BEDS', 'BATHS', 'LOCATION', 'SQUAREFEET', 'YEARBUILT',
'HOA/MONTH', 'LATITUDE', 'LONGITUDE', 'AggregateCrime','PRICE']]
#Pre-covid prices should offset price increase in 2021/22 and show us pricing more in-line with
#what we would have expected before covid and inflation
train_data=X[X['SOLDDATE']<7]
test_data=X[X['SOLDDATE']>=6]
X_train = train_data.drop(['PRICE'], axis = 1)
X_test = test_data.drop(['PRICE'], axis = 1)
y_train = train_data["PRICE"]
y_test = test_data["PRICE"]
In [98]:
# Create the parameter grid based on the results of random search
param_grid = {
'bootstrap': [True],
'max_depth': [80, 90, 100, 110],
'max_features': [2, 3],
'min_samples_leaf': [3, 4, 5],
'min_samples_split': [8, 10, 12],
'n_estimators': [100, 200, 300, 1000]
}
In [99]:
# Use GridSearchCV to search for best hyperparameters
# First create the base model to tune
random_forest = RandomForestRegressor(random_state=RANDOM_SEED)
# Using 3 fold cross validation
rf_grid = GridSearchCV(estimator = random_forest, param_grid = param_grid,
cv = 3, n_jobs = -1, verbose = 2)
# Fit the random search model
rf_grid.fit(X_train, y_train)
Out[99]:
In [100]:
rf_grid.best_params_
Out[100]:
In [101]:
rf = RandomForestRegressor(n_estimators= 300,
min_samples_split=8,
min_samples_leaf = 3,
max_depth = 80,
max_features = 3,
bootstrap = True,
random_state = RANDOM_SEED)
rf.fit(X_train, y_train)
Out[101]:
In [102]:
y_pred_rf = rf.predict(X_test)
In [103]:
print('RMSE Score:',rmse(y_test, y_pred_rf))
In [104]:
rf.score(X_train, y_train)
Out[104]:
In [105]:
get_cv_scores(rf)
Modeling For Sale Data¶
In [106]:
dfp = pd.read_excel ('ForSale05222022.xlsx')
In [107]:
dfp.info()
In [108]:
dfp.drop_duplicates(keep='first',inplace=True)
dfp=dfp.drop(['Source.Name','SALE TYPE','STATE OR PROVINCE','STATUS',
'NEXT OPEN HOUSE START TIME',
'NEXT OPEN HOUSE END TIME',
'FAVORITE',
'INTERESTED',
'MLS#',
'STATUS',
'STATE OR PROVINCE',
'URL (SEE https://www.redfin.com/buy-a-home/comparative-market-analysis FOR INFO ON PRICING)',#has missing data
'SOURCE',
'$/SQUARE FEET'
], axis=1)
# remove special character as well as spaces in between column names
# removing them helps later in processing
dfp.columns = dfp.columns.str.replace(' ', '')
In [109]:
dfp = pd.merge(dfp, crimeCountindex, how='left', left_on='LOCATION', right_on='Community Area')
dfp.head()
Out[109]:
In [110]:
dfp=dfp.drop(['DAYSONMARKET','LOTSIZE','Community Area'],axis=1)
dfp['AggregateCrime']=dfp['AggregateCrime'].fillna(dfp['AggregateCrime'].median())
In [111]:
# converting the sold date field to a proper date time format
df['SOLDDATE'] = pd.to_datetime(df['SOLDDATE'])
df['SOLDDATE']=pd.DatetimeIndex(df['SOLDDATE']).year
df.dropna(subset = ['SOLDDATE'], inplace=True)
df['SOLDDATE']=df['SOLDDATE'].astype(int)
In [112]:
dfp['CITY'] = dfp.CITY.str.upper()
dfp = dfp[dfp['CITY'] != 'ARLINGTON HEIGHTS']
In [113]:
#Mapping data to numerical values
mapping_dict = {"PROPERTYTYPE":{
"Condo/Co-op": 1,
"Single Family Residential": 2,
"Townhouse": 3,
"Multi-Family (2-4 Unit)": 4,
"Multi-Family (5+ Unit)": 5,
"Other": 6,
"Vacant Land": 7,
"Parking":8
},
"CITY":{
'CHICAGO' : 1,
'OAK PARK' : 0,
'OAK LAWN' : 0,
'BERWYN' : 0,
'BURBANK' : 0,
'EVERGREEN PARK' : 0,
'ELMWOOD PARK' : 0,
'FOREST PARK' : 0,
'CICERO' : 0,
'RIVER FOREST' : 0,
'HOMETOWN' : 0,
'STICKNEY' : 0,
'LINCOLNWOOD' : 0,
'HARWOOD HEIGHTS' : 0,
'BRIDGEVIEW' : 0,
'SKOKIE' : 0,
'NORRIDGE' : 0,
'RIVERSIDE' : 0,
'NORTH RIVERSIDE' : 0,
'RIVER GROVE' : 0,
'BURNHAM' : 0,
'FOREST VIEW' : 0,
'RIVERDALE' : 0,
'CALUMET PARK' : 0,
'MELROSE PARK' : 0,
'NOTTINGHAM PARK' : 0,
'LAWNDALE' : 0,
'MAYWOOD' : 0,
'NILES' : 0},
"SOLDDATE":{
2016: 1,
2017: 2,
2018: 3,
2019: 4,
2020: 5,
2021: 6,
2022: 7
}
}
df = df.replace(mapping_dict)
In [114]:
dfp['PROPERTYTYPE'] = dfp['PROPERTYTYPE'].astype('str')
dfp['CITY'] = dfp['CITY'].astype('str')
dfp['LOCATION'] = dfp['LOCATION'].astype('str')
dfp['SOLDDATE'] = 7
dfp['SOLDDATE']=dfp['SOLDDATE'].astype(int)
catmap = preprocessing.LabelEncoder()
dfp['PROPERTYTYPE'] = catmap.fit_transform(dfp['PROPERTYTYPE'].values)
dfp['CITY'] = catmap.fit_transform(dfp['CITY'].values)
dfp['LOCATION'] = catmap.fit_transform(dfp['LOCATION'].values)
dfp['ZIPORPOSTALCODE'] = catmap.fit_transform(dfp['ZIPORPOSTALCODE'].values)
dfp['YEARBUILT'] = catmap.fit_transform(dfp['YEARBUILT'].values)
In [115]:
from sklearn.impute import SimpleImputer, KNNImputer
knniputer = KNNImputer()
knniputer.fit(dfp[["SQUAREFEET"]])
dfp["SQUAREFEET"] = knniputer.transform(dfp[["SQUAREFEET"]])
dfp["SQUAREFEET"] = dfp["SQUAREFEET"].apply(lambda x: int(x))
In [116]:
dfp['HOA/MONTH'] = dfp['HOA/MONTH'].fillna(0)
In [117]:
total = dfp.isnull().sum().sort_values(ascending=False)
percent_1 = dfp.isnull().sum()/dfp.isnull().count()*100
percent_2 = (round(percent_1, 1)).sort_values(ascending=False)
missing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%'])
missing_data.head(33)
Out[117]:
In [118]:
x_pred = dfp.loc[:, ['SOLDDATE', 'PROPERTYTYPE', 'CITY', 'ZIPORPOSTALCODE', 'BEDS', 'BATHS', 'LOCATION', 'SQUAREFEET', 'YEARBUILT',
'HOA/MONTH', 'LATITUDE', 'LONGITUDE', 'AggregateCrime']]
y_pred = dfp["PRICE"]
In [119]:
y_actual_predictions = rf.predict(x_pred)
dff = pd.DataFrame({'ActualPrice': y_pred, 'PredictedPrice': y_actual_predictions})
# Reversing log normalization
#dff['PredictedPrice'] = dff['PredictedPrice'].apply(lambda x: np.exp(x))
#dff['ActualPrice'] = dff['ActualPrice'].apply(lambda x: np.exp(x))
dff=pd.concat([dff,x_pred], axis=1)
dff['PERCENTAGE']=round(((dff['PredictedPrice']-dff['ActualPrice'])/dff['ActualPrice'])*100, 2)
In [120]:
dff['PERCENTAGE'].value_counts()
Out[120]:
In [121]:
filter_values = [ -100, 0, 50, 100]
fig = plt.figure()
fig, ax = plt.subplots()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(True)
ax.spines['bottom'].set_visible(False)
feq=dff['PERCENTAGE'].value_counts(bins=filter_values).sort_index()
feq.plot.barh(figsize=(8,8), color="red", width=0.9)
plt.title("Percentage Difference", fontsize=20)
for i, v in enumerate(feq):
ax.text(v + 1, i - 0.1, str(v), color="black", fontweight='bold')
ax.set_xticks([])
plt.show()
In [122]:
badbuys=pd.DataFrame(dff[['LATITUDE', 'LONGITUDE']][dff['PERCENTAGE']<50])
goodbuys=pd.DataFrame(dff[['LATITUDE', 'LONGITUDE']][dff['PERCENTAGE']>=50])
goodbuyslocationlist = goodbuys.values.tolist()
badbuyslocationlist = badbuys.values.tolist()
In [123]:
map = folium.Map(location=[41.85, -87.68], zoom_start=12)
for point in range(0, len(goodbuyslocationlist)):
folium.Marker(goodbuyslocationlist[point], icon=folium.Icon(color='green', icon='home')).add_to(map)
map
Out[123]:
In [124]:
map = folium.Map(location=[41.85, -87.68], zoom_start=12)
for point in range(0, len(badbuyslocationlist)):
folium.Marker(badbuyslocationlist[point], icon=folium.Icon(color='red', icon='home')).add_to(map)
map
Out[124]:
Costliest House¶
In [125]:
df.dropna(subset = ['PRICE'], inplace=True)
df['PRICE']=df['PRICE'].astype('int')
num_list=[ "PRICE"]
for i in num_list:
df=df[df[i].between(outlier_lower_iqr(i), outlier_upper_iqr(i))]
highest=pd.DataFrame(df.nlargest(5, 'PRICE'))
costliest=pd.DataFrame(highest[['LATITUDE', 'LONGITUDE']])
costliestlist = costliest.values.tolist()
In [126]:
map = folium.Map(location=[41.85, -87.68], zoom_start=12)
for point in range(0, len(costliestlist)):
folium.Marker(costliestlist[point], icon=folium.Icon(color='red', icon='home')).add_to(map)
map
Out[126]:
In [ ]: